# -*- coding: utf-8 -*-
import json
from time import sleep
import requests
from config import get_names, save_data, connect
from Logs.log import log1


class shopeeSpider:
    def __init__(self):
        self.ua = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36"
        self.sessions = requests.session()
        self.retryUrls = []
        self.dt = ""

    def req_url(self, url, params=None):
        headers = {
            'user-agent': self.ua,
        }
        try:
            resp = self.sessions.get(url, headers=headers, params=params)
            if resp.status_code == 200:
                return json.loads(resp.text)
        except:
            return

    def main(self, username, num):
        try:
            url = "https://shopee.co.id/api/v2/shop/get?username={}".format(username)
            list_url = "https://shopee.co.id/api/v2/search_items/"
            detail_url = "https://shopee.co.id/api/v2/item/get"
            star_url = "https://shopee.co.id/api/v2/item/get_ratings"
            resp = self.req_url(url)
            shopid = resp.get("data").get("shopid")
            shopName = resp.get("data").get("name")
            list_params = {
                "match_id": shopid,
                "order": "desc",
                "page_type": "shop",
                "newest": 0,
                "limit": 30
            }
            list_resp = self.req_url(list_url, list_params)
            if not list_resp:
                log1.error("{}列表页请求失败".format(username))
                if num == 1:
                    self.retryUrls.append(username)
                return True
            total_count = list_resp.get("total_count")
            if not total_count:
                log1.info("{}: total_count为空".format(username))
                if num == 1:
                    self.retryUrls.append(username)
                return True
            total_page = total_count // 30 + 1
            for page in range(total_page):
                try:
                    list_params["newest"] = page * 30
                    list_resp = self.req_url(list_url, list_params)
                    items = list_resp.get("items")
                    for item in items:
                        itemid = ""
                        try:
                            itemid = item.get("itemid")
                            if not itemid:
                                continue
                            detail_params = {
                                "itemid": itemid,
                                "shopid": shopid
                            }
                            detail_resp = self.req_url(detail_url, detail_params)
                            detail_item = detail_resp.get("item")
                            name = detail_item.get("name").replace("'", "").replace("' ", "' ")
                            rating_star = detail_item.get("item_rating").get("rating_star")
                            rating_star = round(float(rating_star), 2)
                            sold = detail_item.get("cmt_count")
                            historical_sold = detail_item.get("historical_sold")
                            price = detail_item.get("price")
                            price = float(price) / 100000
                            stock = detail_item.get("stock")
                            star_resp = self.req_url(star_url, detail_params)
                            stars = star_resp.get("data").get("item_rating_summary").get("rating_count")
                            star_1 = stars[0]
                            star_2 = stars[1]
                            star_3 = stars[2]
                            star_4 = stars[3]
                            star_5 = stars[4]
                            # rating_total = star_resp.get("data").get("item_rating_summary").get("rating_total")  # 总评论数
                            rcount_with_context = star_resp.get("data").get("item_rating_summary").get(
                                "rcount_with_context")  # 有内容评论数
                            # 保存数据
                            save_data(connect,
                                      [str(itemid), str(self.dt), str(shopName), str(name), str(rating_star), str(sold),
                                       str(historical_sold), str(price), str(stock), str(star_5), str(star_4),
                                       str(star_3), str(star_2), str(star_1), str(rcount_with_context)])
                        except Exception as e:
                            log1.error("{}抓取失败: {}".format(itemid, e))
                            continue
                        sleep(1)
                except Exception as e:
                    log1.error("{}第{}页请求失败: {}".format(shopid, page + 1, e))
                    continue
            sleep(3)
        except Exception as e:
            log1.error("{}店铺页加载异常: {}".format(username, e))
            if num == 1:
                self.retryUrls.append(username)

    def run(self, usernames):
        import datetime
        # 获取当前年月日
        d = datetime.date.today()
        self.dt = d.strftime('%Y%m%d')
        for username in usernames:
            username = username[0]
            self.main(username, 1)
        if self.retryUrls:
            for retryUrl in self.retryUrls:
                self.main(retryUrl, 0)
                sleep(5)


if __name__ == '__main__':
    # usernames = (("tntbeautyshop",), ("storyofmu",), ("distributor.creamrd.ori",), ("makeupbeautyhouse",), ("beautycosindo_kr",))
    usernames = get_names()
    sg = shopeeSpider()
    sg.run(usernames)